{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''\n",
    "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'prepare/mesolitica-tpu.json'\n",
    "b2_application_key_id = os.environ['b2_application_key_id']\n",
    "b2_application_key = os.environ['b2_application_key']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "client = storage.Client()\n",
    "bucket = client.bucket('mesolitica-tpu-general')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "best = '1020200'\n",
    "directory = 't5-tiny-summary'\n",
    "!rm -rf output out {directory}\n",
    "!mkdir {directory}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = best\n",
    "\n",
    "blob = bucket.blob(f'{directory}/model.ckpt-{model}.data-00000-of-00002')\n",
    "blob.download_to_filename(f'{directory}/model.ckpt-{model}.data-00000-of-00002')\n",
    "\n",
    "blob = bucket.blob(f'{directory}/model.ckpt-{model}.data-00001-of-00002')\n",
    "blob.download_to_filename(f'{directory}/model.ckpt-{model}.data-00001-of-00002')\n",
    "\n",
    "blob = bucket.blob(f'{directory}/model.ckpt-{model}.index')\n",
    "blob.download_to_filename(f'{directory}/model.ckpt-{model}.index')\n",
    "\n",
    "blob = bucket.blob(f'{directory}/model.ckpt-{model}.meta')\n",
    "blob.download_to_filename(f'{directory}/model.ckpt-{model}.meta')\n",
    "\n",
    "blob = bucket.blob(f'{directory}/checkpoint')\n",
    "blob.download_to_filename(f'{directory}/checkpoint')\n",
    "\n",
    "blob = bucket.blob(f'{directory}/operative_config.gin')\n",
    "blob.download_to_filename(f'{directory}/operative_config.gin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'{directory}/checkpoint', 'w') as fopen:\n",
    "    fopen.write(f'model_checkpoint_path: \"model.ckpt-{model}\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from b2sdk.v1 import *\n",
    "info = InMemoryAccountInfo()\n",
    "b2_api = B2Api(info)\n",
    "application_key_id = b2_application_key_id\n",
    "application_key = b2_application_key\n",
    "b2_api.authorize_account(\"production\", application_key_id, application_key)\n",
    "file_info = {'how': 'good-file'}\n",
    "b2_bucket = b2_api.get_bucket_by_name('malaya-model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<b2sdk.file_version.FileVersionInfo at 0x7fbe80662da0>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tar = 't5-tiny-summarization-2021-07-31.tar.gz'\n",
    "os.system(f'tar -czvf {tar} {directory}')\n",
    "outPutname = f'finetuned/{tar}'\n",
    "b2_bucket.upload_local_file(\n",
    "    local_file=tar,\n",
    "    file_name=outPutname,\n",
    "    file_infos=file_info,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.system(f'rm {tar}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "import t5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = t5.models.MtfModel(\n",
    "    model_dir=directory,\n",
    "    tpu=None,\n",
    "    tpu_topology=None,\n",
    "    model_parallelism=1,\n",
    "    batch_size=1,\n",
    "    sequence_length={\"inputs\": 512, \"targets\": 512},\n",
    "    learning_rate_schedule=0.003,\n",
    "    save_checkpoints_steps=5000,\n",
    "    keep_checkpoint_max=3,\n",
    "    iterations_per_loop=100,\n",
    "    mesh_shape=\"model:1,batch:1\", \n",
    "    mesh_devices=[\"cpu:0\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf output/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_model_dir': 't5-tiny-summary', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "isolate_session_state: true\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': None, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=100, num_shards=None, num_cores_per_replica=1, per_host_input_for_training=4, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu_cluster_resolver.TPUClusterResolver object at 0x7fbd6c74d6a0>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n"
     ]
    }
   ],
   "source": [
    "import gin\n",
    "\n",
    "from t5.data import sentencepiece_vocabulary\n",
    "\n",
    "DEFAULT_SPM_PATH = 'prepare/sp10m.cased.ms-en.model'\n",
    "DEFAULT_EXTRA_IDS = 100\n",
    "model_dir = directory\n",
    "\n",
    "def get_default_vocabulary():\n",
    "    return sentencepiece_vocabulary.SentencePieceVocabulary(\n",
    "      DEFAULT_SPM_PATH, DEFAULT_EXTRA_IDS)\n",
    "\n",
    "with gin.unlock_config():\n",
    "    gin.parse_config_file(t5.models.mtf_model._operative_config_path(model_dir))\n",
    "    gin.bind_parameter(\"Bitransformer.decode.beam_size\", 1)\n",
    "    gin.bind_parameter(\"Bitransformer.decode.temperature\", 0)\n",
    "    gin.bind_parameter(\"utils.get_variable_dtype.slice_dtype\", \"float32\")\n",
    "    gin.bind_parameter(\n",
    "        \"utils.get_variable_dtype.activation_dtype\", \"float32\")\n",
    "    \n",
    "vocabulary = t5.data.SentencePieceVocabulary(DEFAULT_SPM_PATH)\n",
    "estimator = model.estimator(vocabulary, disable_tpu=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1020200, 'model.ckpt-1020200', 't5-tiny-summary/model.ckpt-1020200')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "checkpoint_step = t5.models.mtf_model._get_latest_checkpoint_from_dir(model_dir)\n",
    "model_ckpt = \"model.ckpt-\" + str(checkpoint_step)\n",
    "checkpoint_path = os.path.join(model_dir, model_ckpt)\n",
    "checkpoint_step, model_ckpt, checkpoint_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running infer on CPU\n",
      "INFO:tensorflow:feature inputs : Tensor(\"Reshape:0\", shape=(1, 512), dtype=int32)\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/mesh_tensorflow/transformer/utils.py:427: Print (from tensorflow.python.ops.logging_ops) is deprecated and will be removed after 2018-08-20.\n",
      "Instructions for updating:\n",
      "Use tf.print instead of tf.Print. Note that tf.print returns a no-output operator that directly prints the output. Outside of defuns or eager mode, this operator will not be executed unless it is directly specified in session.run or used as a control dependency for other operators. This is only a concern in graph mode. Below is an example of how to ensure tf.print executes in graph mode:\n",
      "\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/k                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/o                  size 294912       slice_size 294912       Shape[heads=768, d_model=384]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/q                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias size 384          slice_size 384          Shape[heads=12, buckets=32]                                 \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/v                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/k                size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/o                size 294912       slice_size 294912       Shape[heads=768, d_model=384]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/q                size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/v                size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wi/kernel         size 516096       slice_size 516096       Shape[d_model=384, d_ff=1344]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wo/kernel         size 516096       slice_size 516096       Shape[d_ff=1344, d_model=384]                               \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/k                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/o                  size 294912       slice_size 294912       Shape[heads=768, d_model=384]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/q                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/v                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/k                size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/o                size 294912       slice_size 294912       Shape[heads=768, d_model=384]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/q                size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/v                size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wi/kernel         size 516096       slice_size 516096       Shape[d_model=384, d_ff=1344]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wo/kernel         size 516096       slice_size 516096       Shape[d_ff=1344, d_model=384]                               \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/k                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/o                  size 294912       slice_size 294912       Shape[heads=768, d_model=384]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/q                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/v                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/k                size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/o                size 294912       slice_size 294912       Shape[heads=768, d_model=384]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/q                size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/v                size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wi/kernel         size 516096       slice_size 516096       Shape[d_model=384, d_ff=1344]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wo/kernel         size 516096       slice_size 516096       Shape[d_ff=1344, d_model=384]                               \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/k                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/o                  size 294912       slice_size 294912       Shape[heads=768, d_model=384]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/q                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/v                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/k                size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/o                size 294912       slice_size 294912       Shape[heads=768, d_model=384]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/q                size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/v                size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wi/kernel         size 516096       slice_size 516096       Shape[d_model=384, d_ff=1344]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wo/kernel         size 516096       slice_size 516096       Shape[d_ff=1344, d_model=384]                               \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable decoder/final_layer_norm/scale                               size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/k                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/o                  size 294912       slice_size 294912       Shape[heads=768, d_model=384]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/q                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias size 384          slice_size 384          Shape[heads=12, buckets=32]                                 \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/v                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wi/kernel         size 516096       slice_size 516096       Shape[d_model=384, d_ff=1344]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wo/kernel         size 516096       slice_size 516096       Shape[d_ff=1344, d_model=384]                               \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/k                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/o                  size 294912       slice_size 294912       Shape[heads=768, d_model=384]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/q                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/v                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wi/kernel         size 516096       slice_size 516096       Shape[d_model=384, d_ff=1344]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wo/kernel         size 516096       slice_size 516096       Shape[d_ff=1344, d_model=384]                               \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/k                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/o                  size 294912       slice_size 294912       Shape[heads=768, d_model=384]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/q                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/v                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wi/kernel         size 516096       slice_size 516096       Shape[d_model=384, d_ff=1344]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wo/kernel         size 516096       slice_size 516096       Shape[d_ff=1344, d_model=384]                               \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/k                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/o                  size 294912       slice_size 294912       Shape[heads=768, d_model=384]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/q                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/v                  size 294912       slice_size 294912       Shape[d_model=384, heads=768]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wi/kernel         size 516096       slice_size 516096       Shape[d_model=384, d_ff=1344]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wo/kernel         size 516096       slice_size 516096       Shape[d_ff=1344, d_model=384]                               \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/layer_norm/scale                 size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable encoder/final_layer_norm/scale                               size 384          slice_size 384          Shape[d_model=384]                                          \n",
      "INFO:tensorflow:Variable shared/embedding                                             size 12337152     slice_size 12337152     Shape[vocab=32128, d_model=384]                             \n",
      "INFO:tensorflow:Trainable Variables            count: 89      Total size: 34759680         Total slice_size: 34759680       \n",
      "INFO:tensorflow:All Variables                  count: 89      Total size: 34759680         Total slice_size: 34759680       \n",
      "INFO:tensorflow:Counters:\n",
      "einsum: 3.61e+10\n",
      "einsum_unique: 3.61e+10\n",
      "output: 6.09e+08\n",
      " output/AddOperation: 1.48e+08\n",
      " output/BinaryOpWithBroadcasting: 5.25e+06\n",
      " output/Constant: 3.15e+06\n",
      " output/EinsumOperation: 1.28e+08\n",
      " output/ImportOperation: 551\n",
      " output/MinMaxOperation: 3.15e+06\n",
      " output/OneHotOperation: 1e+08\n",
      " output/RangeOperation: 1.02e+03\n",
      " output/ReduceOperation: 1.59e+05\n",
      " output/ReshapeOperation: 2.54e+07\n",
      " output/ScalarAddOperation: 4.21e+06\n",
      " output/ScalarMultiplyOperation: 9.65e+06\n",
      " output/ShiftOperation: 512\n",
      " output/SlicewiseOperation: 1.07e+08\n",
      " output/StopGradient: 3.77e+07\n",
      " output/Variable: 3.48e+07\n",
      " output/WhileLoopOperation: 3.15e+06\n",
      "output_unique: 6.09e+08\n",
      " output_unique/AddOperation: 1.48e+08\n",
      " output_unique/BinaryOpWithBroadcasting: 5.25e+06\n",
      " output_unique/Constant: 3.15e+06\n",
      " output_unique/EinsumOperation: 1.28e+08\n",
      " output_unique/ImportOperation: 551\n",
      " output_unique/MinMaxOperation: 3.15e+06\n",
      " output_unique/OneHotOperation: 1e+08\n",
      " output_unique/RangeOperation: 1.02e+03\n",
      " output_unique/ReduceOperation: 1.59e+05\n",
      " output_unique/ReshapeOperation: 2.54e+07\n",
      " output_unique/ScalarAddOperation: 4.21e+06\n",
      " output_unique/ScalarMultiplyOperation: 9.65e+06\n",
      " output_unique/ShiftOperation: 512\n",
      " output_unique/SlicewiseOperation: 1.07e+08\n",
      " output_unique/StopGradient: 3.77e+07\n",
      " output_unique/Variable: 3.48e+07\n",
      " output_unique/WhileLoopOperation: 3.15e+06\n",
      "variables: 3.48e+07\n",
      " variables/trainable: 3.48e+07\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/array_ops.py:1475: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/saved_model/signature_def_utils_impl.py:201: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from t5-tiny-summary/model.ckpt-1020200\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: output/temp-b'1627724030'/saved_model.pb\n"
     ]
    }
   ],
   "source": [
    "from mesh_tensorflow.transformer import dataset as transformer_dataset\n",
    "\n",
    "def serving_input_fn():\n",
    "    inputs = tf.placeholder(\n",
    "            dtype=tf.string,\n",
    "            shape=[None],\n",
    "            name=\"inputs\")\n",
    "\n",
    "    batch_size = tf.shape(inputs)[0]\n",
    "    padded_inputs = tf.pad(inputs, [(0, tf.mod(-tf.size(inputs), batch_size))])\n",
    "    dataset = tf.data.Dataset.from_tensor_slices(padded_inputs)\n",
    "    dataset = dataset.map(lambda x: {\"inputs\": x})\n",
    "    dataset = transformer_dataset.encode_all_features(dataset, vocabulary)\n",
    "    dataset = transformer_dataset.pack_or_pad(\n",
    "        dataset=dataset,\n",
    "        length=model._sequence_length,\n",
    "        pack=False,\n",
    "        feature_keys=[\"inputs\"]\n",
    "    )\n",
    "    dataset = dataset.batch(tf.cast(batch_size, tf.int64))\n",
    "    features = tf.data.experimental.get_single_element(dataset)\n",
    "    return tf.estimator.export.ServingInputReceiver(\n",
    "        features=features, receiver_tensors=inputs)\n",
    "\n",
    "out = estimator.export_saved_model('output', serving_input_fn, checkpoint_path=checkpoint_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-16-5b89b6a20c22>:7: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.\n",
      "INFO:tensorflow:Restoring parameters from output/1627724030/variables/variables\n"
     ]
    }
   ],
   "source": [
    "config = tf.ConfigProto()\n",
    "config.allow_soft_placement = True\n",
    "sess = tf.Session(config = config)\n",
    "meta_graph_def = tf.saved_model.loader.load(\n",
    "        sess,\n",
    "        [tf.saved_model.tag_constants.SERVING],\n",
    "        out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tiny-summarization/model.ckpt'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'tiny-summarization/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "strings = [\n",
    "    n.name\n",
    "    for n in tf.get_default_graph().as_graph_def().node\n",
    "    if ('encoder' in n.op\n",
    "    or 'decoder' in n.name\n",
    "    or 'shared' in n.name\n",
    "    or 'inputs' in n.name\n",
    "    or 'output' in n.name\n",
    "    or 'SentenceTokenizer' in n.name\n",
    "    or 'self/Softmax' in n.name)\n",
    "    and 'adam' not in n.name\n",
    "    and 'Assign' not in n.name\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names,\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from tiny-summarization/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-19-504c79665720>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 144 variables.\n",
      "INFO:tensorflow:Converted 144 variables to const ops.\n",
      "4840 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('tiny-summarization', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import struct\n",
    "\n",
    "unknown = b'\\xff\\xff\\xff\\xff'\n",
    "\n",
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "        \n",
    "    for node in graph_def.node:\n",
    "        \n",
    "        if node.op == 'RefSwitch':\n",
    "          node.op = 'Switch'\n",
    "          for index in xrange(len(node.input)):\n",
    "            if 'moving_' in node.input[index]:\n",
    "              node.input[index] = node.input[index] + '/read'\n",
    "        elif node.op == 'AssignSub':\n",
    "          node.op = 'Sub'\n",
    "          if 'use_locking' in node.attr: del node.attr['use_locking']\n",
    "        elif node.op == 'AssignAdd':\n",
    "          node.op = 'Add'\n",
    "          if 'use_locking' in node.attr: del node.attr['use_locking']\n",
    "        elif node.op == 'Assign':\n",
    "          node.op = 'Identity'\n",
    "          if 'use_locking' in node.attr: del node.attr['use_locking']\n",
    "          if 'validate_shape' in node.attr: del node.attr['validate_shape']\n",
    "          if len(node.input) == 2:\n",
    "            node.input[0] = node.input[1]\n",
    "            del node.input[1]\n",
    "            \n",
    "        if 'Reshape/shape' in node.name or 'Reshape_1/shape' in node.name:\n",
    "            b = node.attr['value'].tensor.tensor_content\n",
    "            arr_int = [int.from_bytes(b[i:i + 4], 'little') for i in range(0, len(b), 4)]\n",
    "            if len(arr_int):\n",
    "                arr_byte = [unknown] + [struct.pack('<i', i) for i in arr_int[1:]]\n",
    "                arr_byte = b''.join(arr_byte)\n",
    "                node.attr['value'].tensor.tensor_content = arr_byte\n",
    "            \n",
    "            if len(node.attr['value'].tensor.int_val):\n",
    "                node.attr['value'].tensor.int_val[0] = -1\n",
    "    \n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('tiny-summarization/frozen_model.pb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'import/inputs:0' shape=(?,) dtype=string>,\n",
       " <tf.Tensor 'import/SelectV2_3:0' shape=(?, 512) dtype=int32>)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i = g.get_tensor_by_name('import/inputs:0')\n",
    "o = g.get_tensor_by_name('import/SelectV2_3:0')\n",
    "i, o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sess = tf.Session(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = \"\"\"\n",
    "Amanah Kedah berpendapat jika ada Adun Pakatan Harapan atau Bersatu negeri itu mahu berpaling tadah memberikan sokongan kepada kumpulan Muafakat Nasional, mereka perku membuat kenyataan rasmi mengenainya.\n",
    "\n",
    "Pengerusi Amanah Kedah, Phahrolrazi Mohd Zawawi, berkata disebabkan tiada mana-mana Adun membuat kenyataan berhubung isu itu maka kerajaan negeri berpendapat tiada apa-apa yang berlaku.\n",
    "\n",
    "Ditemui media selepas mengadakan pertemuan tertutup lebih sejam dengan Menteri Besar, Mukhriz Mahathir, hari ini Phahrolrazi berkata pihaknya juga mendapati kerajaan negeri masih berfungsi seperti biasa.\n",
    "\n",
    "\"Kami bincang keadaan semasa, ada juga kita sentuh (cubaan menukar kerajaan negeri), tetapi kita lihat kerajaan masih berfungsi.\n",
    "\n",
    "\"Tidak ada apa-apa kenyataan dari pihak sana (pembangkang) bahawa mereka sudah cukup majoriti setakat ini,\" katanya seperti dipetik BH Online.\n",
    "\n",
    "Spekulasi mengenai pertukaran kerajaan menjadi kencang sejak semalam ekoran berlaku pertemuan tertutup pemimpin PAS dan Umno Kedah di Alor Setar semalam.\n",
    "\n",
    "Turut hadir Setiausaha Agung PAS yang juga Menteri di Jabatan Perdana Menteri, Takiyuddin Hassan, dan Menteri Besar Terengganu, Dr Ahmad Samsuri Mokhtar.  \n",
    "\n",
    "Cuba jatuhkan sejak dulu\n",
    "\n",
    "Perkembangan itu berlaku kesan tindakan PKR memecat dan menggantung sejumlah anggota mereka baru-baru ini dan dipercayai memberi kesan terhadap pendirian wakil rakyat parti itu di Kedah.\n",
    "\n",
    "Turut disebut-sebut akan beralih arah dalam perjalanan politik mereka ialah Adun Bersatu.\n",
    "\n",
    "Untuk rekod berdasarkan pecahan parti PAS menguasai kerusi terbesar dalam DUN dan lazimnya pemimpin parti itu akan menjadi pilihan menjadi menteri besar jika berlaku pertukaran kerajaan.\n",
    "\n",
    "Menurut Phahrolrazi, jika ada mana-mana wakil rakyat Bersatu atau PH mahu melompat, mereka wajar menyatakannya secara rasmi.\n",
    "\n",
    "Tanpa kenyataan begitu, katanya, Amanah beranggapan isu perubahan kerajaan negeri masih bersifat spekulasi.\n",
    "\n",
    "Timbalan Pengerusi Amanah Kedah, Dr Ismail Salleh, pula berkata ada kemungkinan Adun Bersatu, PH atau exco negeri tu yang sudah diumpan untuk membelakangkan mandat rakyat.\n",
    "\n",
    "Beliau yang juga exco Kedah berkata memang sejak dulu lagi PAS cuba menjatuhkan kerajaan negeri dengan memujuk Adun PH serta Bersatu bertindak seperti rakan mereka di Perak, Johor dan Selangor.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "# minimum cleaning, just simply to remove newlines.\n",
    "def cleaning(string):\n",
    "    string = string.replace('\\n', ' ')\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    return string\n",
    "\n",
    "string = cleaning(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.3 s, sys: 4.35 s, total: 14.7 s\n",
      "Wall time: 7.7 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1, 512)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "o_ = test_sess.run(o, feed_dict = {i: [f'ringkasan: {string}']})\n",
    "o_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sentencepiece as spm\n",
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load(DEFAULT_SPM_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Pengerusi Amanah Kedah berpendapat bahawa tiada Adun Pakatan Harapan atau Bersatu mahu berpaling tadah. Pengerusi Amanah menyatakan kerana 'kami mendapati tiada Adun Pakatan Harapan' untuk berfungsi. Sementara itu Mukhriz rupanya bertemu pemberita bersama pemimpin PAS negeri.\n"
     ]
    }
   ],
   "source": [
    "for k in range(len(o_)):\n",
    "    print(k, sp_model.DecodeIds(o_[k].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(minimum_size=1536000)',\n",
    "             #'quantize_weights(fallback_min=-10240, fallback_max=10240)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-32-8e582c316c2f>:3: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.gfile.GFile.\n"
     ]
    }
   ],
   "source": [
    "pb = 'tiny-summarization/frozen_model.pb'\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "    \n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "       ['inputs'],\n",
    "       ['SelectV2_3'], transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'import/inputs:0' shape=(?,) dtype=string>,\n",
       " <tf.Tensor 'import/SelectV2_3:0' shape=(?, 512) dtype=int32>)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = load_graph('tiny-summarization/frozen_model.pb.quantized')\n",
    "i = g.get_tensor_by_name('import/inputs:0')\n",
    "o = g.get_tensor_by_name('import/SelectV2_3:0')\n",
    "i, o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sess = tf.InteractiveSession(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<b2sdk.file_version.FileVersionInfo at 0x7fbd10171e48>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = 'tiny-summarization/frozen_model.pb.quantized'\n",
    "outPutname = 'abstractive-summarization-v2/tiny-t5-quantized/model.pb'\n",
    "b2_bucket.upload_local_file(\n",
    "    local_file=file,\n",
    "    file_name=outPutname,\n",
    "    file_infos=file_info,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<b2sdk.file_version.FileVersionInfo at 0x7fbd461fe630>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = 'tiny-summarization/frozen_model.pb'\n",
    "outPutname = 'abstractive-summarization-v2/tiny-t5/model.pb'\n",
    "b2_bucket.upload_local_file(\n",
    "    local_file=file,\n",
    "    file_name=outPutname,\n",
    "    file_infos=file_info,\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
